I don't actually want to encode into tokens for my use case, quickly count to check my request won't exceed the limit.

This should be faster since we don't initialize the memory for the output array.

```
const crypto = require('crypto');
// Generate a random string of a given length
function generateRandomString(length) {
    return crypto.randomBytes(length).toString('hex');
}

const {encode, decode, countTokens} = require('gpt-3-encoder')

let str = 'This is an example sentence to try encoding out on!'
// let now = Date.now();
let encoded = encode(str)
console.log('Encoded this string looks like: ', encoded)
console.log('We can look at each token and what it represents)
let tokencount = 0;
for(let token of encoded){
    tokencount ++;
    console.log({token, string: decode([token])})
}
console.log("there are n tokens: ", tokencount);
let decoded = decode(encoded)
console.log('We can decode it back into:\n', decoded)

let now = Date.now();
// todo: write an benchmark for the above method vs  int countTokens(str)
str = generateRandomString(10000);

console.time('fencode');
encoded = encode(str);
console.log(`First encode to cache string n stuff in mem`);
console.timeEnd('fencode');


console.log(`Original string length: ${str.length}`);
// Benchmark the encode function
console.time('encode');
encoded = encode(str);
console.log(`Encoded string length: ${encoded.length}`);
console.timeEnd('encode');

// Benchmark the countTokens function
console.time('countTokens');
let tokenCount = countTokens(str);
console.log(`Number of tokens: ${tokenCount}`);
console.timeEnd('countTokens');


console.log(`Original string length: ${str.length}`);
console.log(`Encoded string length: ${encoded.length}`);
console.log(`Number of tokens: ${tokenCount}`);
```

```
We can decode it back into:
 This is an example sentence to try encoding out on!
First encode to cache string n stuff in mem
fencode: 163.57ms
Original string length: 20000
Encoded string length: 11993
encode: 124.265ms
Number of tokens: 11993
countTokens: 29.2ms
Original string length: 20000
Encoded string length: 11993
Number of tokens: 11993

```

Co-authored-by: Andrew Healey <[email protected]>

I don't actually want to encode into tokens for my use case, quickly count to check my request won't exceed the limit.

This should be faster since we don't initialize the memory for the output array.

```
const crypto = require('crypto');
// Generate a random string of a given length
function generateRandomString(length) {
    return crypto.randomBytes(length).toString('hex');
}

const {encode, decode, countTokens} = require('gpt-3-encoder')

let str = 'This is an example sentence to try encoding out on!'
// let now = Date.now();
let encoded = encode(str)
console.log('Encoded this string looks like: ', encoded)
console.log('We can look at each token and what it represents)
let tokencount = 0;
for(let token of encoded){
    tokencount ++;
    console.log({token, string: decode([token])})
}
console.log("there are n tokens: ", tokencount);
let decoded = decode(encoded)
console.log('We can decode it back into:\n', decoded)

let now = Date.now();
// todo: write an benchmark for the above method vs  int countTokens(str)
str = generateRandomString(10000);

console.time('fencode');
encoded = encode(str);
console.log(`First encode to cache string n stuff in mem`);
console.timeEnd('fencode');


console.log(`Original string length: ${str.length}`);
// Benchmark the encode function
console.time('encode');
encoded = encode(str);
console.log(`Encoded string length: ${encoded.length}`);
console.timeEnd('encode');

// Benchmark the countTokens function
console.time('countTokens');
let tokenCount = countTokens(str);
console.log(`Number of tokens: ${tokenCount}`);
console.timeEnd('countTokens');


console.log(`Original string length: ${str.length}`);
console.log(`Encoded string length: ${encoded.length}`);
console.log(`Number of tokens: ${tokenCount}`);
```

```
We can decode it back into:
 This is an example sentence to try encoding out on!
First encode to cache string n stuff in mem
fencode: 163.57ms
Original string length: 20000
Encoded string length: 11993
encode: 124.265ms
Number of tokens: 11993
countTokens: 29.2ms
Original string length: 20000
Encoded string length: 11993
Number of tokens: 11993

```

Co-authored-by: Kier <[email protected]>

# Conflicts:
#	Encoder.js

merge back the one other change and follow nick with moving to version 1.2 for feature add.

…e updates

maybe we should host jsdoc on github pages

also performed npm audit and updates for security vulnerabilities.

It seems to work fine

By using the Buffer class in this way, you can encode and decode strings as UTF-8 without using the TextEncoder and TextDecoder classes. This can be useful if you need to support earlier versions of Node.js that do not have these classes in the util module.

…eaking change but i think its actually mor widely compatible and should be fine.

…by webpaking... todo need to test browser version

This precomputes bpe_ranks so that it is it the from needed make sure to rebuild if you change the bpe dump file